import requests
from lxml import etree
import csv
import os
import time
import json
import pandas as pd
import re
import django
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'VehicleViewPanel.settings')
django.setup()
from myApp.models import CarInfomation


class spider(object):
    def __init__(self):
        self.spiderName = 'VehicleView'
        self.spiderUrl= "https://www.dongchedi.com/motor/pc/car/rank_data?aid=1839&app_name=auto_web_pc&city_name=%E8%8B%8F%E5%B7%9E&count=10&month=&new_energy_type=&rank_data_type=11&brand_id=&price=&manufacturer=&outter_detail_type=&nation=0"
        #反爬
        self.header={
            'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36'
        }


    def init(self):
        if not os.path.exists('./temp.csv'):
            car_data = {
                "brand": "品牌",
                "carname": "汽车名称",
                "carimg": "汽车图片链接",
                "salevolume": "销售量",
                "price": "价格",
                "manufacturer": "制造商",
                "rank": "排名",
                "carmodel": "车型",
                "energytype": "能源类型",
                "marketime": "上市时间",
                "insure": "保修信息"
            }
            row_data = [
                car_data["brand"],
                car_data["carname"],
                car_data["carimg"],
                car_data["salevolume"],
                car_data["price"],
                car_data["manufacturer"],
                car_data["rank"],
                car_data["carmodel"],
                car_data["energytype"],
                car_data["marketime"],
                car_data["insure"]
            ]
            with open('./temp.csv', 'a', newline='', encoding='utf-8') as wf:
                writer = csv.writer(wf)
                writer.writerow(["brand","carname","carimg","salevolume","price","manufacturer","rank","carmodel","energytype","marketime","insure"])


    def get_page(self):
        with open('./spiderPage.txt', 'r', encoding='utf-8') as r_f:
            return r_f.readlines()[-1].strip()


    def set_page(self, newPage):
        with open('./spiderPage.txt', 'a') as a_f:

            a_f.write('\n' + str(newPage))



    def main(self):
        count=self.get_page()
        params={
            'offset':int(count)
        }
        print("数据从{}开始爬取".format(int(count)+1))
        pageJson=requests.get(self.spiderUrl, headers=self.header,params=params).json()
        pageJson=pageJson["data"]["list"]
        for index,car in enumerate(pageJson):
            carData=[]
            print("正在爬取{}行".format(index + 1))
            #print(car["brand_name"],car["series_name"],car["image"],car["count"],car["max_price"],car["min_price"],car["sub_brand_name"],car["rank"])
            #
            carData.append(car["brand_name"])
            #
            carData.append(car["series_name"])
            #
            carData.append(car["image"])
            #
            carData.append(car["count"])
            #价格
            price=[]
            price.append(car["max_price"])
            price.append(car["min_price"])
            carData.append(price)
            #厂商
            carData.append(car["sub_brand_name"])
            #排名
            carData.append(car["rank"])
            #车型



            #第二个页面信息获取
            carNum=car["series_id"]

            infoHTML=requests.get("https://www.dongchedi.com/auto/params-carIds-x-%s"%carNum,headers=self.header)
            infoHTMLpath=etree.HTML(infoHTML.text)
            carModel=infoHTMLpath.xpath("//div[@data-row-anchor='jb']/div[2]/div/text()")[0]
            carData.append(carModel)



            #能源类型
            energyType=infoHTMLpath.xpath("//div[@data-row-anchor='fuel_form']/div[2]/div/text()")[0]

            carData.append(energyType)


            #上市时间


            #marketTime=infoHTMLpath.xpath("//div[@data-row-anchor='market_time']/div[2]/div/text()")[0]
            #print(marketTime)
            #carData.append(marketTime)
            try:
                # 尝试获取保险信息
                marketTime = infoHTMLpath.xpath("//div[@data-row-anchor='market_time']/div[2]/div/text()")[0]

                carData.append(marketTime)
            except IndexError:
                # 如果找不到值，将保险信息设置为空字符串并继续
                carData.append(" ")




            #保修期限
            #insure=infoHTMLpath.xpath("//div[@data-row-anchor='period']/div[2]/div/text()")[0]

            # if infoHTMLpath.xpath("//div[@data-row-anchor='period']/div[2]/div/text()")[0]:
            #     insure = infoHTMLpath.xpath("//div[@data-row-anchor='period']/div[2]/div/text()")[0]
            # else:
            #     insure = 0
            try:
                # 尝试获取保险信息
                insure = infoHTMLpath.xpath("//div[@data-row-anchor='period']/div[2]/div/text()")[0]
                carData.append(insure)
            except IndexError:
                # 如果找不到值，将保险信息设置为空字符串并继续
                carData.append(" ")

           # carData.append(insure)

            print(carData)
            self.save_to_csv(carData)



        # print(pageJson)
        #
        self.set_page(int(count)+10)
        self.main()


    def save_to_csv(self,resultData):
        with open('./temp.csv', 'a', newline='', encoding='utf-8') as f:
            writer = csv.writer(f)
            writer.writerow(resultData)



    #数据清洗
    def clear_csv(self):
        df=pd.read_csv('./temp.csv')

        #如果是有空值，就删除这一行
        df.dropna(inplace=True)
        #删除数据重复值
        df.drop_duplicates(inplace=True)
        print("总共%d"%df.shape[0])
        return df.values


    def save_to_sql(self):
        data=self.clear_csv()
        for car in data:
            CarInfomation.objects.create(
                brand=car[0],
                carname=car[1],
                carimg=car[2],
                salevolume=car[3],
                price=car[4],
                manufacturer=car[5],
                rank=car[6],
                carmodel=car[7],
                energytype=car[8],
                marketime=car[9],
                insure=car[10]
                )

if __name__ == '__main__':
    spiderObj = spider()
    #spiderObj.init()
    #spiderObj.main()
    spiderObj.save_to_sql()